package main

import (
	"fmt"
	"gitee.com/zackeus/goutil/x/encodes/hashutil"
	"github.com/pkoukk/tiktoken-go"
	tiktoken_loader "github.com/pkoukk/tiktoken-go-loader"
)

func testTiktoken() {
	text := "我去你吗的"

	// 如果你不想在运行时下载字典，你可以使用离线加载器
	tiktoken.SetBpeLoader(tiktoken_loader.NewOfflineLoader())
	//tiktoken.NewTiktoken()
	tke, err := tiktoken.GetEncoding(tiktoken.MODEL_R50K_BASE)
	if err != nil {
		fmt.Printf("getEncoding: %v", err)
		return
	}

	// encode
	token := tke.Encode(text, nil, nil)

	// 输出tokens和token数量
	fmt.Println("Tokens:", token)
	fmt.Println("Number of tokens:", len(token))
}

func main() {
	println(hashutil.Hash(hashutil.AlgoSHA256, ""))
}
